<!DOCTYPE html>
<html lang="zh-CN">
    <head>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <meta name="robots" content="noodp" />
        <meta http-equiv="X-UA-Compatible" content="IE=edge, chrome=1">
        <title>python分析豆瓣影评 - 海蓝的博客</title><meta name="Description" content="运维搬砖"><meta property="og:title" content="python分析豆瓣影评" />
<meta property="og:description" content="本文主要对当前热门剧《隐秘的角落》进行豆瓣短评以及评价进行可视化分析，数据抓取主要为python编写的爬虫。" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://guohailan.github.io/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" /><meta property="og:image" content="https://guohailan.github.io/logo.png"/><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2020-07-05T13:59:11&#43;00:00" />
<meta property="article:modified_time" content="2020-07-05T13:59:11&#43;00:00" />

<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="https://guohailan.github.io/logo.png"/>

<meta name="twitter:title" content="python分析豆瓣影评"/>
<meta name="twitter:description" content="本文主要对当前热门剧《隐秘的角落》进行豆瓣短评以及评价进行可视化分析，数据抓取主要为python编写的爬虫。"/>
<meta name="application-name" content="LoveIt">
<meta name="apple-mobile-web-app-title" content="LoveIt"><meta name="theme-color" content="#ffffff"><meta name="msapplication-TileColor" content="#da532c"><link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" />
        <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
        <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5"><link rel="manifest" href="/site.webmanifest"><link rel="canonical" href="https://guohailan.github.io/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" /><link rel="prev" href="https://guohailan.github.io/redis%E9%9B%86%E7%BE%A4%E6%90%AD%E5%BB%BA/" /><link rel="next" href="https://guohailan.github.io/kubeadm%E6%90%AD%E5%BB%BAk8s-1-18/" /><link rel="stylesheet" href="/lib/normalize/normalize.min.css"><link rel="stylesheet" href="/css/style.min.css"><link rel="stylesheet" href="/lib/fontawesome-free/all.min.css"><link rel="stylesheet" href="/lib/animate/animate.min.css"><script type="application/ld+json">
    {
        "@context": "http://schema.org",
        "@type": "BlogPosting",
        "headline": "python分析豆瓣影评",
        "inLanguage": "zh-CN",
        "mainEntityOfPage": {
            "@type": "WebPage",
            "@id": "https:\/\/guohailan.github.io\/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84\/"
        },"genre": "posts","keywords": "可视化, 爬虫","wordcount":  2348 ,
        "url": "https:\/\/guohailan.github.io\/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84\/","datePublished": "2020-07-05T13:59:11+00:00","dateModified": "2020-07-05T13:59:11+00:00","license": "This work is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License.","publisher": {
            "@type": "Organization",
            "name": "","logo": "https:\/\/guohailan.github.io\/images\/avatar.png"},"author": {
                "@type": "Person",
                "name": "guohailan"
            },"description": ""
    }
    </script></head>
    <body header-desktop="fixed" header-mobile="auto"><script type="text/javascript">(window.localStorage && localStorage.getItem('theme') ? localStorage.getItem('theme') === 'dark' : ('auto' === 'auto' ? window.matchMedia('(prefers-color-scheme: dark)').matches : 'auto' === 'dark')) && document.body.setAttribute('theme', 'dark');</script>

        <div id="mask"></div><div class="wrapper"><header class="desktop" id="header-desktop">
    <div class="header-wrapper">
        <div class="header-title">
            <a href="/" title="海蓝的博客"><span class="header-title-pre"><i class='far fas fa-blog fa-fw'></i></span><span id="id-1" class="typeit"></span></a>
        </div>
        <div class="menu">
            <div class="menu-inner"><a class="menu-item" href="/posts/"> 所有文章 </a><a class="menu-item" href="/tags/"> 标签 </a><a class="menu-item" href="/categories/"> 分类 </a><a class="menu-item" href="https://github.com/guohailan" title="GitHub" rel="noopener noreffer" target="_blank"><i class='fab fa-github fa-fw'></i>  </a><span class="menu-item delimiter"></span><span class="menu-item search" id="search-desktop">
                        <input type="text" placeholder="搜索文章标题或内容..." id="search-input-desktop">
                        <a href="javascript:void(0);" class="search-button search-toggle" id="search-toggle-desktop" title="搜索">
                            <i class="fas fa-search fa-fw"></i>
                        </a>
                        <a href="javascript:void(0);" class="search-button search-clear" id="search-clear-desktop" title="清空">
                            <i class="fas fa-times-circle fa-fw"></i>
                        </a>
                        <span class="search-button search-loading" id="search-loading-desktop">
                            <i class="fas fa-spinner fa-fw fa-spin"></i>
                        </span>
                    </span><a href="javascript:void(0);" class="menu-item theme-switch" title="切换主题">
                    <i class="fas fa-adjust fa-fw"></i>
                </a>
            </div>
        </div>
    </div>
</header><header class="mobile" id="header-mobile">
    <div class="header-container">
        <div class="header-wrapper">
            <div class="header-title">
                <a href="/" title="海蓝的博客"><span class="header-title-pre"><i class='far fas fa-blog fa-fw'></i></span><span id="id-2" class="typeit"></span></a>
            </div>
            <div class="menu-toggle" id="menu-toggle-mobile">
                <span></span><span></span><span></span>
            </div>
        </div>
        <div class="menu" id="menu-mobile"><div class="search-wrapper">
                    <div class="search mobile" id="search-mobile">
                        <input type="text" placeholder="搜索文章标题或内容..." id="search-input-mobile">
                        <a href="javascript:void(0);" class="search-button search-toggle" id="search-toggle-mobile" title="搜索">
                            <i class="fas fa-search fa-fw"></i>
                        </a>
                        <a href="javascript:void(0);" class="search-button search-clear" id="search-clear-mobile" title="清空">
                            <i class="fas fa-times-circle fa-fw"></i>
                        </a>
                        <span class="search-button search-loading" id="search-loading-mobile">
                            <i class="fas fa-spinner fa-fw fa-spin"></i>
                        </span>
                    </div>
                    <a href="javascript:void(0);" class="search-cancel" id="search-cancel-mobile">
                        取消
                    </a>
                </div><a class="menu-item" href="/posts/" title="">所有文章</a><a class="menu-item" href="/tags/" title="">标签</a><a class="menu-item" href="/categories/" title="">分类</a><a class="menu-item" href="https://github.com/guohailan" title="GitHub" rel="noopener noreffer" target="_blank"><i class='fab fa-github fa-fw'></i></a><a href="javascript:void(0);" class="menu-item theme-switch" title="切换主题">
                <i class="fas fa-adjust fa-fw"></i>
            </a></div>
    </div>
</header>
<div class="search-dropdown desktop">
    <div id="search-dropdown-desktop"></div>
</div>
<div class="search-dropdown mobile">
    <div id="search-dropdown-mobile"></div>
</div>
<main class="main">
                <div class="container"><div class="toc" id="toc-auto">
            <h2 class="toc-title">目录</h2>
            <div class="toc-content" id="toc-content-auto"></div>
        </div><article class="page single"><h1 class="single-title animated flipInX">python分析豆瓣影评</h1><div class="post-meta">
            <div class="post-meta-line"><span class="post-author"><a href="/" title="Author" rel=" author" class="author"><i class="fas fa-user-circle fa-fw"></i>guohailan</a></span>&nbsp;<span class="post-category">收录于 <a href="/categories/python/"><i class="far fa-folder fa-fw"></i>Python</a></span></div>
            <div class="post-meta-line"><i class="far fa-calendar-alt fa-fw"></i>&nbsp;<time datetime="2020-07-05">2020-07-05</time>&nbsp;<i class="fas fa-pencil-alt fa-fw"></i>&nbsp;约 2348 字&nbsp;
                <i class="far fa-clock fa-fw"></i>&nbsp;预计阅读 5 分钟&nbsp;<span id="/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" class="leancloud_visitors" data-flag-title="python分析豆瓣影评">
                        <i class="far fa-eye fa-fw"></i>&nbsp;<span class=leancloud-visitors-count></span>&nbsp;次阅读
                    </span>&nbsp;</div>
        </div><div class="featured-image"><img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="/images/thumbnail/spider.png"
        data-srcset="/images/thumbnail/spider.png, /images/thumbnail/spider.png 1.5x, /images/thumbnail/spider.png 2x"
        data-sizes="auto"
        alt="/images/thumbnail/spider.png"
        title="/images/thumbnail/spider.png" /></div><div class="details toc" id="toc-static"  kept="">
                <div class="details-summary toc-title">
                    <span>目录</span>
                    <span><i class="details-icon fas fa-angle-right"></i></span>
                </div>
                <div class="details-content toc-content" id="toc-content-static"><nav id="TableOfContents">
  <ul>
    <li><a href="#数据抓取">数据抓取</a>
      <ul>
        <li><a href="#日志打印">日志打印</a></li>
        <li><a href="#爬虫">爬虫</a></li>
      </ul>
    </li>
    <li><a href="#可视化">可视化</a></li>
    <li><a href="#主函数">主函数</a></li>
    <li><a href="#完整源码">完整源码</a></li>
  </ul>
</nav></div>
            </div><div class="content" id="content"><p>本文主要对当前热门剧《隐秘的角落》进行豆瓣短评以及评价进行可视化分析，数据抓取主要为python编写的爬虫。</p>
<p>本文主要分为2个部分，分别对爬虫和可视化部分进行详解，本文脚本基于python 3.8.0版本进行编写</p>
<table>
<thead>
<tr>
<th>第三方依赖包</th>
<th>说明</th>
</tr>
</thead>
<tbody>
<tr>
<td>pyecharts</td>
<td>百度开源可视化工具，用于生成图表和词云</td>
</tr>
<tr>
<td>jieba</td>
<td>国内比较好用的分词工具</td>
</tr>
<tr>
<td>requests</td>
<td>python常用HTTP 库</td>
</tr>
<tr>
<td>lxml</td>
<td>pyhon常用XML和HTML解析库</td>
</tr>
</tbody>
</table>
<p><figure><a class="lightgallery" href="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/score.png" title="/images/python分析豆瓣影评/score.png" data-thumbnail="/images/python分析豆瓣影评/score.png" data-sub-html="<h2> </h2>">
        <img
            class="lazyload"
            src="/svg/loading.min.svg"
            data-src="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/score.png"
            data-srcset="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/score.png, /images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/score.png 1.5x, /images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/score.png 2x"
            data-sizes="auto"
            alt="/images/python分析豆瓣影评/score.png" />
    </a><figcaption class="image-caption"> </figcaption>
    </figure></p>
<p><figure><a class="lightgallery" href="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/short_comments.png" title="/images/python分析豆瓣影评/short_comments.png" data-thumbnail="/images/python分析豆瓣影评/short_comments.png" data-sub-html="<h2> </h2>">
        <img
            class="lazyload"
            src="/svg/loading.min.svg"
            data-src="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/short_comments.png"
            data-srcset="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/short_comments.png, /images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/short_comments.png 1.5x, /images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/short_comments.png 2x"
            data-sizes="auto"
            alt="/images/python分析豆瓣影评/short_comments.png" />
    </a><figcaption class="image-caption"> </figcaption>
    </figure></p>
<h2 id="数据抓取">数据抓取</h2>
<h3 id="日志打印">日志打印</h3>
<p>为了方便调试和问题定位，单独引入了logging模块进行日志打印，为方便复用，日志部分单独写到了一个文件中，创建log.py文件</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">logging.handlers</span>

<span class="k">class</span> <span class="nc">Logger</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="n">level_relations</span> <span class="o">=</span> <span class="p">{</span>
        <span class="s1">&#39;debug&#39;</span><span class="p">:</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">,</span>
        <span class="s1">&#39;info&#39;</span><span class="p">:</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">,</span>
        <span class="s1">&#39;warning&#39;</span><span class="p">:</span><span class="n">logging</span><span class="o">.</span><span class="n">WARNING</span><span class="p">,</span>
        <span class="s1">&#39;error&#39;</span><span class="p">:</span><span class="n">logging</span><span class="o">.</span><span class="n">ERROR</span><span class="p">,</span>
        <span class="s1">&#39;crit&#39;</span><span class="p">:</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span>
    <span class="p">}</span><span class="c1">#日志级别关系映射</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="n">filename</span><span class="p">,</span><span class="n">level</span><span class="o">=</span><span class="s1">&#39;info&#39;</span><span class="p">,</span><span class="n">when</span><span class="o">=</span><span class="s1">&#39;H&#39;</span><span class="p">,</span><span class="n">backCount</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span><span class="n">fmt</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">%(asctime)s</span><span class="s1"> - </span><span class="si">%(pathname)s</span><span class="s1">[line:</span><span class="si">%(lineno)d</span><span class="s1">] - </span><span class="si">%(levelname)s</span><span class="s1">: </span><span class="si">%(message)s</span><span class="s1">&#39;</span><span class="p">):</span>
        <span class="c1">#日志格式</span>
        <span class="c1">#----------------------------</span>
        <span class="c1"># %(asctime)s      年-月-日 时-分-秒，毫秒</span>
        <span class="c1"># %(filename)s     文件名，不含目录</span>
        <span class="c1"># %(pathname)s     目录名，完整路径</span>
        <span class="c1"># %(funcName)s     函数名</span>
        <span class="c1"># %(levelname)s    级别名</span>
        <span class="c1"># %(lineno)d       行号</span>
        <span class="c1"># %(module)s       模块名</span>
        <span class="c1"># %(message)s      日志信息</span>
        <span class="c1"># %(name)s         日志模块名</span>
        <span class="c1"># %(process)d      进程id</span>
        <span class="c1"># %(processName)s  进程名</span>
        <span class="c1"># %(thread)d       线程id</span>
        <span class="c1"># %(threadName)s   线程名</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="n">filename</span><span class="p">)</span>
        <span class="n">format_str</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">Formatter</span><span class="p">(</span><span class="n">fmt</span><span class="p">)</span><span class="c1">#设置日志格式</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">level_relations</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">level</span><span class="p">))</span><span class="c1">#设置日志级别</span>
        <span class="n">sh</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">StreamHandler</span><span class="p">()</span><span class="c1">#往屏幕上输出</span>
        <span class="n">sh</span><span class="o">.</span><span class="n">setFormatter</span><span class="p">(</span><span class="n">format_str</span><span class="p">)</span> <span class="c1">#设置屏幕上显示的格式</span>
        <span class="n">th</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">handlers</span><span class="o">.</span><span class="n">TimedRotatingFileHandler</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span><span class="n">when</span><span class="o">=</span><span class="n">when</span><span class="p">,</span><span class="n">backupCount</span><span class="o">=</span><span class="n">backCount</span><span class="p">,</span><span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span><span class="c1">#往文件里写入#指定间隔时间自动生成文件的处理器</span>
        <span class="c1">#实例化TimedRotatingFileHandler</span>
        <span class="c1">#interval是时间间隔，backupCount是备份文件的个数，如果超过这个个数，就会自动删除，when是间隔的时间单位，单位有以下几种：</span>
        <span class="c1"># S 秒</span>
        <span class="c1"># M 分</span>
        <span class="c1"># H 小时、</span>
        <span class="c1"># D 天、</span>
        <span class="c1"># W 每星期（interval==0时代表星期一）</span>
        <span class="c1"># midnight 每天凌晨</span>
        <span class="n">th</span><span class="o">.</span><span class="n">setFormatter</span><span class="p">(</span><span class="n">format_str</span><span class="p">)</span><span class="c1">#设置文件里写入的格式</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">sh</span><span class="p">)</span> <span class="c1">#把对象加到logger里</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">th</span><span class="p">)</span>
</code></pre></td></tr></table>
</div>
</div><h3 id="爬虫">爬虫</h3>
<p>豆瓣对未登陆账号的请求有些限制，短评只能看到前面200条，所以采取了登陆账号的方式来请求更多的数据。豆瓣返回的数据为xml，因此采用了xpath的方式来获取短评数据，然后将数据存入本地文本中</p>
<p><figure><a class="lightgallery" href="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/xpath.png" title="/images/python分析豆瓣影评/xpath.png" data-thumbnail="/images/python分析豆瓣影评/xpath.png" data-sub-html="<h2> </h2>">
        <img
            class="lazyload"
            src="/svg/loading.min.svg"
            data-src="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/xpath.png"
            data-srcset="/images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/xpath.png, /images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/xpath.png 1.5x, /images/python%e5%88%86%e6%9e%90%e8%b1%86%e7%93%a3%e5%bd%b1%e8%af%84/xpath.png 2x"
            data-sizes="auto"
            alt="/images/python分析豆瓣影评/xpath.png" />
    </a><figcaption class="image-caption"> </figcaption>
    </figure></p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="k">class</span> <span class="nc">sprider</span><span class="p">():</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="c1"># 设置请求头</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">:</span> <span class="s1">&#39;Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko&#39;</span><span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">short_comments_file</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">+</span><span class="s2">&#34;</span><span class="se">\\</span><span class="s2">Short_comments_file.txt&#34;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">score_file</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">+</span><span class="s2">&#34;</span><span class="se">\\</span><span class="s2">score_file.txt&#34;</span>

    <span class="k">def</span> <span class="nf">login</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="n">session</span><span class="p">):</span>
        <span class="n">login_url</span> <span class="o">=</span> <span class="s1">&#39;https://accounts.douban.com/j/mobile/login/basic&#39;</span>
        <span class="c1">## 账号和密码需要修改为正确的</span>
        <span class="n">postData</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;ck&#39;</span><span class="p">:</span> <span class="s1">&#39;&#39;</span><span class="p">,</span>
                    <span class="s1">&#39;name&#39;</span><span class="p">:</span> <span class="s1">&#39;yourname&#39;</span><span class="p">,</span>
                    <span class="s1">&#39;password&#39;</span><span class="p">:</span> <span class="s1">&#39;passwd&#39;</span><span class="p">,</span>
                    <span class="s1">&#39;remember&#39;</span><span class="p">:</span> <span class="s1">&#39;false&#39;</span><span class="p">,</span>
                    <span class="s1">&#39;ticket&#39;</span><span class="p">:</span> <span class="s1">&#39;&#39;</span><span class="p">}</span>
        <span class="c1"># 从网上看到，需要先get请求一次才能成功，不然能登录能200，但是不能进行后续的get请求抛403，具体原因不详</span>
        <span class="n">a</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">login_url</span><span class="p">,</span><span class="n">headers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
        <span class="n">b</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="n">login_url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">postData</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">b</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&#34;登录成功&#34;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="c1"># 登陆失败打印返回码和失败详情</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&#34;登录失败:&#34;</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">b</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">b</span><span class="o">.</span><span class="n">text</span><span class="p">))</span>
            <span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">48</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span><span class="n">url</span><span class="p">,</span><span class="n">session</span><span class="p">):</span>
        <span class="c1"># get请求获取短评详情</span>
        <span class="n">request</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span><span class="n">headers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
            <span class="c1"># 防止乱码，将编码格式设置为utf-8</span>
            <span class="n">request</span><span class="o">.</span><span class="n">encode</span> <span class="o">=</span> <span class="s1">&#39;utf-8&#39;</span>
            <span class="k">return</span> <span class="n">request</span><span class="o">.</span><span class="n">text</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">&#39;current status_code:&#39;</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">text</span><span class="p">))</span>
            <span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">44</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">xpath_analysis</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">,</span> <span class="n">xpath</span><span class="p">):</span>
        <span class="n">html</span> <span class="o">=</span> <span class="n">etree</span><span class="o">.</span><span class="n">HTML</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
        <span class="c1"># 使用xpath进行短评解析</span>
        <span class="n">result</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="n">xpath</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">result</span>

    <span class="k">def</span> <span class="nf">to_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
        <span class="c1"># 采用追加的方式将内容写入文件</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">&#39;a&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="c1"># xpath获取的data为列表，所以用遍历的方式写入</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
                <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">init_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">):</span>
        <span class="c1"># 初始化文件，直接清空</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s1">&#39;w&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;&#39;</span><span class="p">)</span>
</code></pre></td></tr></table>
</div>
</div><h2 id="可视化">可视化</h2>
<p>可视化部分主要是对前一步抓取到的数据进行清洗和分析，分词库用的是<a href="https://github.com/fxsjy/jieba" target="_blank" rel="noopener noreffer">jieba</a>。图表生成使用的是百度开源可视化工具<a href="http://pyecharts.herokuapp.com/" target="_blank" rel="noopener noreffer">pyecharts</a>，本文只用了柱状图和词云，更多高级用法见<a href="http://pyecharts.org/#/zh-cn/intro" target="_blank" rel="noopener noreffer">pyecharts官方文档</a></p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span><span class="lnt">56
</span><span class="lnt">57
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="k">class</span> <span class="nc">visualization</span><span class="p">():</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="c1"># 初始化文件路径</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">short_comments_file</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">+</span><span class="s2">&#34;</span><span class="se">\\</span><span class="s2">Short_comments_file.txt&#34;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">score_file</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">+</span><span class="s2">&#34;</span><span class="se">\\</span><span class="s2">score_file.txt&#34;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">analysis_score</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">evaluate</span> <span class="o">=</span> <span class="p">{}</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">score_file</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">():</span>
                <span class="n">i</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">&#34;[A-Za-z0-9\-\:]&#34;</span><span class="p">,</span> <span class="s2">&#34;&#34;</span><span class="p">,</span> <span class="n">i</span><span class="p">)</span>
                <span class="n">i</span> <span class="o">=</span> <span class="n">i</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>  <span class="c1"># 去掉每行的换行符</span>
                <span class="c1"># 如果这个没出现过，就初始化为1</span>
                <span class="k">if</span> <span class="n">i</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">evaluate</span><span class="p">:</span>
                    <span class="n">evaluate</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
                    <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">evaluate</span><span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="c1"># 如果已经出现过了，就在自加1</span>
                    <span class="n">evaluate</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
                    <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">evaluate</span><span class="p">)</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">evaluate</span><span class="p">)</span>
        <span class="n">bar</span> <span class="o">=</span> <span class="n">Bar</span><span class="p">()</span>
        <span class="n">eva</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">count</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">evaluate</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="n">k</span> <span class="o">!=</span> <span class="s1">&#39;&#39;</span><span class="p">:</span>
                <span class="n">eva</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">k</span><span class="p">)</span>
                <span class="n">count</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
        <span class="n">bar</span><span class="o">.</span><span class="n">add_xaxis</span><span class="p">(</span><span class="n">eva</span><span class="p">)</span>  <span class="c1"># 柱状图x轴</span>
        <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;xaxis&#39;</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">eva</span><span class="p">))</span>
        <span class="n">bar</span><span class="o">.</span><span class="n">add_yaxis</span><span class="p">(</span><span class="s2">&#34;评价&#34;</span><span class="p">,</span> <span class="n">count</span><span class="p">)</span>  <span class="c1"># 柱状图y轴</span>
        <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;yaxis&#39;</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">count</span><span class="p">))</span>
        <span class="n">bar</span><span class="o">.</span><span class="n">set_global_opts</span><span class="p">(</span><span class="n">title_opts</span><span class="o">=</span><span class="n">opts</span><span class="o">.</span><span class="n">TitleOpts</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s2">&#34;隐秘的角落 豆瓣评分&#34;</span><span class="p">))</span>
        <span class="c1"># 生成可视化图表</span>
        <span class="n">bar</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path</span><span class="o">+</span><span class="s2">&#34;</span><span class="se">\\</span><span class="s2">score.html&#34;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">analysis_short_comment</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">cut_words</span> <span class="o">=</span> <span class="s2">&#34;&#34;</span>
        <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">short_comments_file</span><span class="p">,</span> <span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf-8&#39;</span><span class="p">):</span>
            <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
            <span class="c1"># 正则去掉标点等无效的字符，对数据进行清洗</span>
            <span class="n">line</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">&#34;[A-Za-z0-9\：\·\—\，\。\“ \”\....]&#34;</span><span class="p">,</span> <span class="s2">&#34;&#34;</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span>
            <span class="c1"># cut_all=False为精确模式，cut_all=True为全词模式</span>
            <span class="n">seg_list</span> <span class="o">=</span> <span class="n">jieba</span><span class="o">.</span><span class="n">cut</span><span class="p">(</span><span class="n">line</span><span class="p">,</span> <span class="n">cut_all</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
            <span class="n">cut_words</span> <span class="o">+=</span> <span class="p">(</span><span class="s2">&#34; &#34;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">seg_list</span><span class="p">))</span>
        <span class="n">all_words</span> <span class="o">=</span> <span class="n">cut_words</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
        <span class="n">c</span> <span class="o">=</span> <span class="n">Counter</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">all_words</span><span class="p">:</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">x</span> <span class="o">!=</span> <span class="s1">&#39;</span><span class="se">\r\n</span><span class="s1">&#39;</span><span class="p">:</span>
                <span class="n">c</span><span class="p">[</span><span class="n">x</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="n">words</span> <span class="o">=</span> <span class="n">c</span><span class="o">.</span><span class="n">most_common</span><span class="p">(</span><span class="mi">500</span><span class="p">)</span>  <span class="c1"># 输出词频最高的前500词</span>
        <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">words</span><span class="p">)</span>
        <span class="n">wordcloud</span> <span class="o">=</span> <span class="n">WordCloud</span><span class="p">()</span>
        <span class="n">wordcloud</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="s2">&#34;&#34;</span><span class="p">,</span> <span class="n">words</span><span class="p">,</span> <span class="n">word_size_range</span><span class="o">=</span><span class="p">[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span> <span class="n">shape</span><span class="o">=</span><span class="s1">&#39;circle&#39;</span><span class="p">)</span>
        <span class="n">wordcloud</span><span class="o">.</span><span class="n">set_global_opts</span><span class="p">(</span><span class="n">title_opts</span><span class="o">=</span><span class="n">opts</span><span class="o">.</span><span class="n">TitleOpts</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s2">&#34;隐秘的角落 短评&#34;</span><span class="p">))</span>
        <span class="n">wordcloud</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path</span><span class="o">+</span><span class="s2">&#34;</span><span class="se">\\</span><span class="s2">short_comment.html&#34;</span><span class="p">)</span>
</code></pre></td></tr></table>
</div>
</div><h2 id="主函数">主函数</h2>
<p>为了防止被豆瓣封IP，降低了采集频率采取了单线程和每次请求随机sleep 0.1s-4s的方式</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="k">def</span> <span class="nf">run_sprider</span><span class="p">(</span><span class="n">sprider</span><span class="p">):</span>
    <span class="c1"># 初始化文件</span>
    <span class="n">sprider</span><span class="o">.</span><span class="n">init_file</span><span class="p">(</span><span class="n">sprider</span><span class="o">.</span><span class="n">short_comments_file</span><span class="p">)</span>
    <span class="n">sprider</span><span class="o">.</span><span class="n">init_file</span><span class="p">(</span><span class="n">sprider</span><span class="o">.</span><span class="n">score_file</span><span class="p">)</span>
    <span class="c1"># 遍历获取数据</span>
    <span class="n">s</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">session</span><span class="p">()</span>
    <span class="n">sprider</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">page_start</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">500</span><span class="p">,</span> <span class="mi">20</span><span class="p">):</span>  <span class="c1"># 范围从0-500，步长为20,页面上总评论数为500+</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">delay</span> <span class="o">=</span> <span class="nb">round</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mf">0.1</span><span class="p">,</span> <span class="mi">4</span><span class="p">),</span> <span class="mi">1</span><span class="p">)</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;i will sleep:&#39;</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">delay</span><span class="p">)</span><span class="o">+</span><span class="s1">&#39;s&#39;</span><span class="p">)</span>
            <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">delay</span><span class="p">)</span>
            <span class="n">URL</span> <span class="o">=</span> <span class="s1">&#39;https://movie.douban.com/subject/33404425/comments?start={}&amp;limit=20&amp;sort=new_score&amp;status=P&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="n">page_start</span><span class="p">)</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;current_request_url:&#39;</span><span class="o">+</span><span class="n">URL</span><span class="p">)</span>
            <span class="n">x</span> <span class="o">=</span> <span class="n">sprider</span><span class="o">.</span><span class="n">get_request</span><span class="p">(</span><span class="n">URL</span><span class="p">,</span><span class="n">s</span><span class="p">)</span>
            <span class="c1"># 短评的xpath路径</span>
            <span class="n">xpath</span> <span class="o">=</span> <span class="s1">&#39;//*[@id=&#34;comments&#34;]/div[*]/div[2]/p/span/text()&#39;</span>
            <span class="n">short_comment</span> <span class="o">=</span> <span class="n">sprider</span><span class="o">.</span><span class="n">xpath_analysis</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">xpath</span><span class="p">)</span>
            <span class="n">sprider</span><span class="o">.</span><span class="n">to_file</span><span class="p">(</span><span class="n">sprider</span><span class="o">.</span><span class="n">short_comments_file</span><span class="p">,</span> <span class="n">short_comment</span><span class="p">)</span>
            <span class="c1"># 评分的xpath路径</span>
            <span class="n">xpath2</span> <span class="o">=</span> <span class="s1">&#39;//*[@id=&#34;comments&#34;]/div[*]/div[2]/h3/span[2]/span[2]/@title&#39;</span>
            <span class="n">score</span> <span class="o">=</span> <span class="n">sprider</span><span class="o">.</span><span class="n">xpath_analysis</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">xpath2</span><span class="p">)</span>
            <span class="n">sprider</span><span class="o">.</span><span class="n">to_file</span><span class="p">(</span><span class="n">sprider</span><span class="o">.</span><span class="n">score_file</span><span class="p">,</span> <span class="n">score</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
            <span class="n">logger</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;current_page:&#39;</span><span class="o">+</span><span class="n">page_start</span><span class="p">)</span>
            <span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">146</span><span class="p">)</span>


<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&#34;__main__&#34;</span><span class="p">:</span>
    <span class="c1"># 数据抓取</span>
    <span class="n">sprider</span> <span class="o">=</span> <span class="n">sprider</span><span class="p">()</span>
    <span class="n">run_sprider</span><span class="p">(</span><span class="n">sprider</span><span class="p">)</span>
    <span class="c1"># 数据分析和图标生成</span>
    <span class="n">c</span> <span class="o">=</span> <span class="n">visualization</span><span class="p">()</span>
    <span class="n">c</span><span class="o">.</span><span class="n">analysis_score</span><span class="p">()</span>
    <span class="n">c</span><span class="o">.</span><span class="n">analysis_short_comment</span><span class="p">()</span>
</code></pre></td></tr></table>
</div>
</div><h2 id="完整源码">完整源码</h2>
<p><a href="log.py" rel="">log.py</a>
<a href="sprider.py" rel="">sprider.py</a></p></div><div class="post-footer" id="post-footer">
    <div class="post-info">
        <div class="post-info-line">
            <div class="post-info-mod">
                <span>更新于 2020-07-05</span>
            </div>
            <div class="post-info-license"></div>
        </div>
        <div class="post-info-line">
            <div class="post-info-md"><span>
                            <a class="link-to-markdown" href="/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/index.md" target="_blank">阅读原始文档</a>
                        </span></div>
            <div class="post-info-share">
                <span><a href="javascript:void(0);" title="分享到 Twitter" data-sharer="twitter" data-url="https://guohailan.github.io/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" data-title="python分析豆瓣影评" data-hashtags="可视化,爬虫"><i class="fab fa-twitter fa-fw"></i></a><a href="javascript:void(0);" title="分享到 Facebook" data-sharer="facebook" data-url="https://guohailan.github.io/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" data-hashtag="可视化"><i class="fab fa-facebook-square fa-fw"></i></a><a href="javascript:void(0);" title="分享到 Hacker News" data-sharer="hackernews" data-url="https://guohailan.github.io/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" data-title="python分析豆瓣影评"><i class="fab fa-hacker-news fa-fw"></i></a><a href="javascript:void(0);" title="分享到 微博" data-sharer="weibo" data-url="https://guohailan.github.io/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" data-title="python分析豆瓣影评" data-image="/images/thumbnail/spider.png"><i class="fab fa-weibo fa-fw"></i></a><a href="javascript:void(0);" title="分享到 百度" data-sharer="baidu" data-url="https://guohailan.github.io/python%E5%88%86%E6%9E%90%E8%B1%86%E7%93%A3%E5%BD%B1%E8%AF%84/" data-title="python分析豆瓣影评"><i data-svg-src="/lib/simple-icons/icons/baidu.min.svg"></i></a></span>
            </div>
        </div>
    </div>

    <div class="post-info-more">
        <section class="post-tags"><i class="fas fa-tags fa-fw"></i>&nbsp;<a href="/tags/%E5%8F%AF%E8%A7%86%E5%8C%96/">可视化</a>,&nbsp;<a href="/tags/%E7%88%AC%E8%99%AB/">爬虫</a></section>
        <section>
            <span><a href="javascript:void(0);" onclick="window.history.back();">返回</a></span>&nbsp;|&nbsp;<span><a href="/">主页</a></span>
        </section>
    </div>

    <div class="post-nav"><a href="/redis%E9%9B%86%E7%BE%A4%E6%90%AD%E5%BB%BA/" class="prev" rel="prev" title="Redis集群搭建"><i class="fas fa-angle-left fa-fw"></i>Redis集群搭建</a>
            <a href="/kubeadm%E6%90%AD%E5%BB%BAk8s-1-18/" class="next" rel="next" title="kubeadm搭建k8s 1.18">kubeadm搭建k8s 1.18<i class="fas fa-angle-right fa-fw"></i></a></div>
</div>
<div id="comments"><div id="valine" class="comment"></div><noscript>
                Please enable JavaScript to view the comments powered by <a href="https://valine.js.org/">Valine</a>.
            </noscript></div></article></div>
            </main><footer class="footer">
        <div class="footer-container"><div class="footer-line"><i class="far fa-copyright fa-fw"></i><span itemprop="copyrightYear">2020 - 2021</span><span class="author" itemprop="copyrightHolder">&nbsp;<a href="/" target="_blank">guohailan</a></span>&nbsp;|&nbsp;<span class="license"><a rel="license external nofollow noopener noreffer" href="https://creativecommons.org/licenses/by-nc/4.0/" target="_blank">CC BY-NC 4.0</a></span></div>
        </div>
    </footer></div>

        <div id="fixed-buttons"><a href="#" id="back-to-top" class="fixed-button" title="回到顶部">
                <i class="fas fa-arrow-up fa-fw"></i>
            </a><a href="#" id="view-comments" class="fixed-button" title="查看评论">
                <i class="fas fa-comment fa-fw"></i>
            </a>
        </div><link rel="stylesheet" href="/lib/valine/valine.min.css"><link rel="stylesheet" href="/lib/lightgallery/lightgallery.min.css"><link rel="stylesheet" href="/lib/katex/katex.min.css"><link rel="stylesheet" href="/lib/katex/copy-tex.min.css"><script type="text/javascript" src="/lib/valine/Valine.min.js"></script><script type="text/javascript" src="/lib/smooth-scroll/smooth-scroll.min.js"></script><script type="text/javascript" src="/lib/autocomplete/autocomplete.min.js"></script><script type="text/javascript" src="/lib/lunr/lunr.min.js"></script><script type="text/javascript" src="/lib/lunr/lunr.stemmer.support.min.js"></script><script type="text/javascript" src="/lib/lunr/lunr.zh.min.js"></script><script type="text/javascript" src="/lib/lazysizes/lazysizes.min.js"></script><script type="text/javascript" src="/lib/twemoji/twemoji.min.js"></script><script type="text/javascript" src="/lib/lightgallery/lightgallery.min.js"></script><script type="text/javascript" src="/lib/lightgallery/lg-thumbnail.min.js"></script><script type="text/javascript" src="/lib/lightgallery/lg-zoom.min.js"></script><script type="text/javascript" src="/lib/clipboard/clipboard.min.js"></script><script type="text/javascript" src="/lib/sharer/sharer.min.js"></script><script type="text/javascript" src="/lib/typeit/typeit.min.js"></script><script type="text/javascript" src="/lib/katex/katex.min.js"></script><script type="text/javascript" src="/lib/katex/auto-render.min.js"></script><script type="text/javascript" src="/lib/katex/copy-tex.min.js"></script><script type="text/javascript" src="/lib/katex/mhchem.min.js"></script><script type="text/javascript">window.config={"code":{"copyTitle":"复制到剪贴板","maxShownLines":10},"comment":{"valine":{"appId":"fok8VPlm0I0mJLCDA5SMvn1o-gzGzoHsz","appKey":"Ghs1UwIPIu4r6tEHw1jLYSFG","avatar":"mp","el":"#valine","emojiCDN":"https://cdn.jsdelivr.net/npm/emoji-datasource-google@5.0.1/img/google/64/","emojiMaps":{"100":"1f4af.png","alien":"1f47d.png","anger":"1f4a2.png","angry":"1f620.png","anguished":"1f627.png","astonished":"1f632.png","black_heart":"1f5a4.png","blue_heart":"1f499.png","blush":"1f60a.png","bomb":"1f4a3.png","boom":"1f4a5.png","broken_heart":"1f494.png","brown_heart":"1f90e.png","clown_face":"1f921.png","cold_face":"1f976.png","cold_sweat":"1f630.png","confounded":"1f616.png","confused":"1f615.png","cry":"1f622.png","crying_cat_face":"1f63f.png","cupid":"1f498.png","dash":"1f4a8.png","disappointed":"1f61e.png","disappointed_relieved":"1f625.png","dizzy":"1f4ab.png","dizzy_face":"1f635.png","drooling_face":"1f924.png","exploding_head":"1f92f.png","expressionless":"1f611.png","face_vomiting":"1f92e.png","face_with_cowboy_hat":"1f920.png","face_with_hand_over_mouth":"1f92d.png","face_with_head_bandage":"1f915.png","face_with_monocle":"1f9d0.png","face_with_raised_eyebrow":"1f928.png","face_with_rolling_eyes":"1f644.png","face_with_symbols_on_mouth":"1f92c.png","face_with_thermometer":"1f912.png","fearful":"1f628.png","flushed":"1f633.png","frowning":"1f626.png","ghost":"1f47b.png","gift_heart":"1f49d.png","green_heart":"1f49a.png","grimacing":"1f62c.png","grin":"1f601.png","grinning":"1f600.png","hankey":"1f4a9.png","hear_no_evil":"1f649.png","heart":"2764-fe0f.png","heart_decoration":"1f49f.png","heart_eyes":"1f60d.png","heart_eyes_cat":"1f63b.png","heartbeat":"1f493.png","heartpulse":"1f497.png","heavy_heart_exclamation_mark_ornament":"2763-fe0f.png","hole":"1f573-fe0f.png","hot_face":"1f975.png","hugging_face":"1f917.png","hushed":"1f62f.png","imp":"1f47f.png","innocent":"1f607.png","japanese_goblin":"1f47a.png","japanese_ogre":"1f479.png","joy":"1f602.png","joy_cat":"1f639.png","kiss":"1f48b.png","kissing":"1f617.png","kissing_cat":"1f63d.png","kissing_closed_eyes":"1f61a.png","kissing_heart":"1f618.png","kissing_smiling_eyes":"1f619.png","laughing":"1f606.png","left_speech_bubble":"1f5e8-fe0f.png","love_letter":"1f48c.png","lying_face":"1f925.png","mask":"1f637.png","money_mouth_face":"1f911.png","nauseated_face":"1f922.png","nerd_face":"1f913.png","neutral_face":"1f610.png","no_mouth":"1f636.png","open_mouth":"1f62e.png","orange_heart":"1f9e1.png","partying_face":"1f973.png","pensive":"1f614.png","persevere":"1f623.png","pleading_face":"1f97a.png","pouting_cat":"1f63e.png","purple_heart":"1f49c.png","rage":"1f621.png","relaxed":"263a-fe0f.png","relieved":"1f60c.png","revolving_hearts":"1f49e.png","right_anger_bubble":"1f5ef-fe0f.png","robot_face":"1f916.png","rolling_on_the_floor_laughing":"1f923.png","scream":"1f631.png","scream_cat":"1f640.png","see_no_evil":"1f648.png","shushing_face":"1f92b.png","skull":"1f480.png","skull_and_crossbones":"2620-fe0f.png","sleeping":"1f634.png","sleepy":"1f62a.png","slightly_frowning_face":"1f641.png","slightly_smiling_face":"1f642.png","smile":"1f604.png","smile_cat":"1f638.png","smiley":"1f603.png","smiley_cat":"1f63a.png","smiling_face_with_3_hearts":"1f970.png","smiling_imp":"1f608.png","smirk":"1f60f.png","smirk_cat":"1f63c.png","sneezing_face":"1f927.png","sob":"1f62d.png","space_invader":"1f47e.png","sparkling_heart":"1f496.png","speak_no_evil":"1f64a.png","speech_balloon":"1f4ac.png","star-struck":"1f929.png","stuck_out_tongue":"1f61b.png","stuck_out_tongue_closed_eyes":"1f61d.png","stuck_out_tongue_winking_eye":"1f61c.png","sunglasses":"1f60e.png","sweat":"1f613.png","sweat_drops":"1f4a6.png","sweat_smile":"1f605.png","thinking_face":"1f914.png","thought_balloon":"1f4ad.png","tired_face":"1f62b.png","triumph":"1f624.png","two_hearts":"1f495.png","unamused":"1f612.png","upside_down_face":"1f643.png","weary":"1f629.png","white_frowning_face":"2639-fe0f.png","white_heart":"1f90d.png","wink":"1f609.png","woozy_face":"1f974.png","worried":"1f61f.png","yawning_face":"1f971.png","yellow_heart":"1f49b.png","yum":"1f60b.png","zany_face":"1f92a.png","zipper_mouth_face":"1f910.png","zzz":"1f4a4.png"},"enableQQ":false,"highlight":true,"lang":"zh-cn","pageSize":10,"placeholder":"说些什么...","recordIP":true,"serverURLs":"https://fok8vplm.lc-cn-n1-shared.com","visitor":true}},"data":{"id-1":"My blog","id-2":"My blog"},"lightGallery":{"actualSize":false,"exThumbImage":"data-thumbnail","hideBarsDelay":2000,"selector":".lightgallery","speed":400,"thumbContHeight":80,"thumbWidth":80,"thumbnail":true},"math":{"delimiters":[{"display":true,"left":"$$","right":"$$"},{"display":true,"left":"\\[","right":"\\]"},{"display":false,"left":"$","right":"$"},{"display":false,"left":"\\(","right":"\\)"}],"strict":false},"search":{"highlightTag":"em","lunrIndexURL":"/index.json","lunrLanguageCode":"zh","lunrSegmentitURL":"/lib/lunr/lunr.segmentit.js","maxResultLength":10,"noResultsFound":"没有找到结果","snippetLength":50,"type":"lunr"},"twemoji":true,"typeit":{"cursorChar":"|","cursorSpeed":1000,"data":{"id-1":["id-1"],"id-2":["id-2"]},"duration":-1,"speed":100}};</script><script type="text/javascript" src="/js/theme.min.js"></script></body>
</html>
